#include <string>
#include <vector>
#include <algorithm>
#include <time.h>
#include "hse_common.h"
#include "hse_types.h"
#include "structured_index.h"

namespace hse {
using std::string;
using std::vector;
using std::map;

StructuredIndex::StructuredIndex(InvertedList& postings)
:postings_(postings){
}

/**
 IndexFileStructure:
 2 byte: headerlength
   |header->    4 byte:wordcount  4 byte:position offset position 1 byte:startwordlength  startword  
   |vocabulary  1 byte:word1len  word1  1 byte:word2len  word2 ...
   |position offset  4 byte:word1position 4 byte:word2position ...
   |position    4 byte:docid 4 byte:count 4 byte:p1 4byte:p2 ...  4 byte:docid 
 */
//TODO:divide invertedlist to small sub list, distribute it to more small file block
int StructuredIndex::Dump(FILE *f) {
	printf("Begin to dump postings...\n");
	unsigned short	header_length_;
	unsigned char	start_word_length_;
	const char		*start_word_;
	unsigned int	wordcount_;
	//unsigned short	position_offset_;

	map<string, Posting*, StringLess>::iterator i = postings_.begin();
	char *buffer = new char[INDEX_FILE_BUFFER_SIZE];  
	unsigned int bp = 0;
	unsigned int position_offset_position;
	start_word_ = i->second->term.c_str();
	start_word_length_ = i->second->term.size();
	header_length_ = 2 //header length recorder:2 byte
		+1   //startwordlength byte 
	   	+ start_word_length_ //startwordlength
		+ 4	//wordcount
		+ 4; //position offset short
	wordcount_ = postings_.size();
	DEBUG_F(printf("header:headerlength:%d startword:%s wordcount:%d", header_length_, start_word_, wordcount_));

	memcpy(buffer, &header_length_, 2);  //unsigned short
	bp += 2;
	memcpy(buffer + bp, &wordcount_, 4); //unsigned short
	bp += 4;
	bp += 4;   //skip position offset, record it later
	buffer[bp++] = start_word_length_;
	memcpy(buffer + bp, start_word_, start_word_length_);
	bp += start_word_length_;
	
	while (i != postings_.end()) {
		buffer[bp++] = i->second->term.size();
		memcpy(buffer + bp, i->second->term.c_str(), i->second->term.size());
		bp += i->second->term.size();
		i++;
	}

	position_offset_position = bp;
	memcpy(buffer + 2 + 4, &position_offset_position, 4);
	DEBUG_F(printf("header:position_offset_position:%d\n", position_offset_position));

	unsigned int pos = bp + wordcount_ * 4;
	i = postings_.begin();
	LOG_INFO(int finished = -1);
	while (i != postings_.end()) {  //write a POSTING
		LOG_INFO(
			if (++finished % 100 == 0)
				printf("Finished %d/%d postings\n", finished, postings_.size());
		);
		memcpy(buffer + bp, &pos, 4); //sizeof unsigned int
		bp += 4;
		vector<DocPositions*>::iterator j = i->second->doc_positions.begin();
		unsigned int int_temp;
		//LOG_INFO(printf("postitions is:%d\n", i->second->doc_positions.size()));
		while (j != i->second->doc_positions.end()) {  //write a DOC
			int_temp = (*j)->doc_id;
			memcpy(buffer + pos, &(int_temp), 4);  //sizeof unsigned int
			pos += 4;
			int_temp = (*j)->positions.size();
			DEBUG_F(printf("write a count:%d\n", int_temp));
			memcpy(buffer + pos, &(int_temp), 4);  //sizeof unsigned int
			pos += 4;
			vector<int>::iterator pi = (*j)->positions.begin();
			while(pi != (*j)->positions.end()) {  //write a POSITION
				int_temp = *pi;
				memcpy(buffer + pos, &(int_temp), 4);
				DEBUG_F(printf("write pos to:%d:%d\n", pos, *pi));
				pos += 4;
				pi++;
			}
			j++;
		}
		i++;
	}
	int ll = fwrite(buffer, 1, pos, f);
	DEBUG_F(printf("Write file len:%d, provides len:%d\n", ll, pos));
	delete[] buffer;
	return 1;
}

/**
 IndexFileStructure:
 2 byte: headerlength
   |header->    4 byte:wordcount  4 byte:position offset position 1 byte:startwordlength  startword  
   |vocabulary  1 byte:word1len  word1  1 byte:word2len  word2 ...
   |position offset  4 byte:word1position 4 byte:word2position ...
   |position    4 byte:docid 4 byte:count 4 byte:p1 4byte:p2 ...  4 byte:docid 
 */
int StructuredIndex::Load(FILE *f) {
	clock_t start_time = clock();
	LOG_INFO(
		unsigned int	total_posistion = 0;
	);

	unsigned short	header_length_;
	//unsigned char	start_word_length_;
	//const char		*start_word_;
	unsigned int	wordcount_;
	unsigned int	position_offset_;	
	unsigned int buffer_len = INDEX_FILE_BUFFER_SIZE;
	unsigned char *buffer = new unsigned char[buffer_len];  //4MB buffer size
	unsigned int l;

	l = fread(buffer, 1, buffer_len, f); //sizeof unsigned char
	unsigned int bp = 0;
	memcpy(&header_length_, buffer + bp, 2); //sizeof unsigned short
	bp += 2;
	memcpy(&wordcount_, buffer + bp, 4);
	bp += 4; //wordcount
	memcpy(&position_offset_, buffer + bp, 4); //position 
	bp += 4;
	DEBUG_F(printf("Load from file: header_len:%d wordcount:%d position_offset:%d\n", header_length_, wordcount_, position_offset_));

	unsigned int vocabulary_offset = header_length_;
	char word[MAX_TERM_LEN];
	unsigned char word_len;
	unsigned int vp = vocabulary_offset;
	unsigned int cur_position_offset, next_position_offset;
	for (unsigned  i = 0; i < wordcount_; ++i) {
		LOG_INFO(if (i % 5000 == 0) printf("finish %d/%d postings(%d ms)!\n", i, wordcount_, (clock() - start_time)));
		memcpy(&word_len, buffer + vp, 1); //word_len 
		vp += 1;
		memcpy(word, buffer + vp, word_len);
		word[word_len] = '\0';
		vp += word_len;
		string s(word);
		DEBUG_F(printf("Retrive a word:%s,", word));

		map<string, Posting*, StringLess>::iterator i_pos = postings_.find(s);
		Posting *posting;
		if (i_pos == postings_.end()) {
			//Create new posting
			posting = new Posting();
			posting->term = s;
			postings_[s] = posting;
			DEBUG_F(printf(" and create new posting!\n"));
		} else {
			posting = i_pos->second;
			DEBUG_F(printf(" and append exist posting!\n"));
		}

		memcpy(&cur_position_offset, buffer + position_offset_, 4);
		if (i != wordcount_ - 1) {
			memcpy(&next_position_offset, buffer + position_offset_ + 4, 4);
		} else {
 			next_position_offset = l;
		}
		position_offset_ += 4;
		while (cur_position_offset < next_position_offset) {
			unsigned int doc_id, p_count, doc_position;
			DocPositions* doc_positions = new DocPositions();
			memcpy(&doc_id, buffer + cur_position_offset, 4);
			cur_position_offset += 4;
			memcpy(&p_count, buffer + cur_position_offset, 4);
			DEBUG_F(printf("p_count:%d\n", p_count));
			cur_position_offset += 4;
			doc_positions->doc_id = doc_id;
			doc_positions->count = p_count;
			for (unsigned int j = 0; j < p_count; ++j, cur_position_offset += 4) {
				DEBUG_F(printf("read from:%d\n", cur_position_offset));
				memcpy(&doc_position, buffer + cur_position_offset, 4);
				doc_positions->positions.push_back(doc_position);
				DEBUG_F(printf("Retrive a word position:term:%s\t doc_id:%d\t position:%d\n", word, doc_id, doc_position));
				LOG_INFO(total_posistion++);
			}
			posting->doc_positions.push_back(doc_positions);
		}
	}

	delete[] buffer;
	LOG_INFO(printf("\nload %d postings, %d positions from index with %d ms!\n", postings_.size(), total_posistion, (clock() - start_time)));
	return 1;
}

} //endof namespace
